#!/usr/bin/env python
# coding: utf-8

# import urllib
import requests
import codecs
import random
import json
import time
import re
import os

CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
SAVE_PATH = os.path.join(CURRENT_PATH, 'images')
CLIENT = requests.session()


def check_dir(dir):
    if not os.path.exists(dir):
        os.mkdir(dir)


check_dir(SAVE_PATH)


def get_page(url):
    print 'GET', url
    # page = urllib.urlopen(url).read()
    page = CLIENT.get(url).content
    return page.decode('gbk')


def main_page():
    return [
        {'column': 'xinggan', 'name': u'性感'},
        {'column': 'sifang', 'name': u'私房'},
        {'column': 'qingchun', 'name': u'清纯'},
        {'column': 'meizi', 'name': u'萌妹子'},
        {'column': 'xiaoqingxin', 'name': u'小清新'},
        {'column': 'nvshen', 'name': u'女神'},
        {'column': 'qizhi', 'name': u'气质美女'},
        {'column': 'mote', 'name': u'模特'},
        {'column': 'bijini', 'name': u'比基尼'},
        {'column': 'baobei', 'name': u'宝贝'},
        {'column': 'luoli', 'name': u'萝莉'},
        {'column': 'wangluo', 'name': u'网络'},
        {'column': 'rihan', 'name': u'日韩美女'},
        {'column': 'oumei', 'name': u'欧美美女'},
    ]


def column_page(column):
    posts = []
    print 'GET COLUMN', column

    url = 'http://www.meizitu.com/a/%s.html' % column
    pattern = re.compile("""<a target='_blank' href="http://www.meizitu.com/a/(\d+).html"><img src="http://pic.meizitu.com/wp-content/uploads/.+.jpg" alt="(<b>)?([^\<\)]+)(</b>)?"></a>""")
    pattern_next_page = re.compile(u"""<li><a href='([a-z0-9_]+).html'>下一页</a></li>""")

    html = get_page(url)

    all = pattern.findall(html)
    if all:
        for i in all:
            # print i[0], i[2]
            posts.append({'id': i[0], 'name': i[2]})

    next_page = pattern_next_page.findall(html)
    # print next_page
    if next_page:
        next_page = next_page[0]
        posts.extend(column_page(column=next_page))

    return posts


def post_page(post_id):
    print 'GET POST', post_id
    url = 'http://www.meizitu.com/a/%s.html' % post_id
    # <img alt="气质身材俱佳的女神，第1张" src="http://pic.meizitu.com/wp-content/uploads/2016a/03/19/01.jpg" />
    # <img alt="[妹子-蓝精灵与红焗饭]" class="scrollLoading" src="http://pic.meizitu.com/wp-content/uploads/2013a/09/28/01.jpg"  style="width:100%;" title="[妹子-蓝精灵与红焗饭]" /><br />
    # <img alt="非常有气质的一个女孩，第1张"  class="scrollLoading" src="http://pic.meizitu.com/wp-content/uploads/2013a/12/65/01.jpg"    />
    pattern = re.compile("""<img alt="[^"]+"([ ]+class="[^"]+")?[ ]+src="(http://pic.meizitu.com/wp-content/uploads/.+.jpg)"([ ]+style="[^"]+")?([ ]+title="[^"]+")?[ ]+/>""")
    html = get_page(url)
    image_urls = [i for item in pattern.findall(html) for i in item if i.startswith('http')]
    return image_urls


def cache_post():
    all_columns = main_page()
    try:
        with open('cache.json') as f:
            cache = json.load(f)
    except Exception, error:
        print 'Read cache.json error:', error
        cache = {'posts': {}, 'images': {}}

    post_cache = cache['posts']
    for column in all_columns:
        if column['name'] in post_cache:
            continue
        post_cache[column['name']] = column_page(column['column'])
        # print cache
        with codecs.open('cache.json', 'w', 'utf-8') as f:
            json.dump(cache, f, indent=4, ensure_ascii=False)


def cache_images():
    try:
        with open('cache.json') as f:
            cache = json.load(f)
        assert 'posts' in cache and 'images' in cache
    except Exception, error:
        print 'Read cache.json error:', error
        cache = {'posts': {}, 'images': {}}

    post_cache = cache['posts']
    image_cache = cache['images']
    for column, posts in post_cache.items():
        for post in posts:
            post_id = post['id']
            # --------------------------------------------------------
            if image_cache.get(post_id):
                continue
            image_cache[post_id] = post_page(post_id)
            with codecs.open('cache.json', 'w', 'utf-8') as f:
                json.dump(cache, f, indent=4, ensure_ascii=False)


def download(url, title=u'妹子图'):
    print 'Download ', url,
    image_name = url[len('http://pic.meizitu.com/wp-content/uploads/'):].replace('/', '-')
    dir = os.path.join(SAVE_PATH, title)
    check_dir(dir)
    save_path = os.path.join(dir, image_name)
    print 'To', save_path,
    if os.path.exists(save_path):
        print '< File Exist!'
        return
    try:
        # urllib.urlretrieve(url, save_path)
        with open(save_path, 'wb') as f:
            f.write(requests.get(url).content)
        print '< END'
    except Exception, error:
        print '< Error:', error,
    finally:
        random_time = random.randint(300, 3000) / 1000.0
        print 'Sleep', random_time
        time.sleep(random_time)


def download_all():
    try:
        with open('cache.json') as f:
            cache = json.load(f)
        assert 'posts' in cache and 'images' in cache
    except Exception, error:
        print 'Read cache.json error:', error
        cache = {'posts': {}, 'images': {}}

    post_cache = cache['posts']
    image_cache = cache['images']
    for column, posts in post_cache.items():
        for post in posts:
            post_id = post['id']
            # --------------------------------------------------------
            post_name = post['name']
            if post_id not in image_cache:
                print 'POST #%s(%s) NOT CACHED!' % (post_id, post_name)
                continue
            for image_url in image_cache[post_id]:
                download(image_url, u'【%s】%s' % (column, post_name))


def main():
    # print post_page(3928)
    # print post_page(4049)
    # return
    cache_post()
    cache_images()
    download_all()


if __name__ == '__main__':
    # download("http://pic.meizitu.com/wp-content/uploads/2014a/12/26/01.jpg")
    main()
